package com.xiaotu.spider.license.pageprocessor;

import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.commons.lang.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.model.TVPublicModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.HtmlUtil;
import com.xiaotu.common.util.HttpUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 TVPublicPageProcessor
 * @日期 2017年10月09日
 * @作者 王艳龙
 * @功能 电视剧备案公示爬取
 */
@Component("TVPublicPageProcessor")
public class TVPublicPageProcessor extends AUrlPageProcessor
{
	private String start_url = "https://www.baidu.com/";
	
	private static final String TABLE = "<table><tbody>%s</tbody></table>";
    
	@Autowired
    private CacheHandler cacheHandler;

    @SuppressWarnings("unchecked")
    @Override
    public void process(Page page)
    {
        this.urlProcess(page);
    }

    private void urlProcess(Page page){
    	try {
    		Connection conn = HttpUtils.connent("https://dsj.nrta.gov.cn/tims/site/views/applications.shanty?appName=note");
			conn.get();
			Response rs = conn.response();
			String html = rs.body();
			if(StringUtil.isNotEmpty(html)){
				Document documentHtml = Jsoup.parse(html,"utf8");
				Elements linkList = documentHtml.select("table[align=center]").select("tbody tr td").select("a[target=_blank]");
				Set<String> linkSet = new HashSet<String>();
				for(Element linkElementList : linkList){
					String linkUrl = linkElementList.select("a").attr("href");
					if(StringUtils.isNotBlank(linkUrl)){
						linkUrl = linkUrl.replace("/tims/site/", "https://dsj.nrta.gov.cn/tims/site/");
					}
					if(!cacheHandler.hExists(Constants.DataRedisKey.Public_History.toString(),linkUrl)){
						linkSet.add(linkUrl);
					}
				}
					if(!linkSet.isEmpty()){
						this.tvProcess(page,linkSet);
					}
			}
			
		} catch (IOException e) {
			e.printStackTrace();
		}
    }
    
    private void tvProcess(Page page,Set<String> linkSet){
    try {
    	Set<String> tvSet = new HashSet<String>();
		for (String linkUrl : linkSet) {
    	Connection conn = HttpUtils.connent(linkUrl);
		conn.get();
		Response rs = conn.response();
		String html = rs.body();
	if(StringUtil.isNotEmpty(html)){
		Document documentHtml = Jsoup.parse(html,"utf8");
		Html divHtml = new Html(documentHtml.toString());
			
    	List<String> trList = divHtml.xpath
                ("//table[@bordercolorlight='#FFFFFF']/tbody/tr").all();
    	if(trList== null || trList.isEmpty())
    		throw new SpiderException(SpiderExceptionCode.DataInvalideException);
    	int begin = 0;
    	for (String trlist : trList) {
    		Html trObj = new Html(String.format(TABLE, trlist));
    		// 去除表头
			if (begin == 0) {
				begin = 1;
				continue;
			}
			String link = trObj.xpath("//td[2]/a[@target='_blank']/@href").toString();
			if (StringUtils.isEmpty(link))
                throw new SpiderException(SpiderExceptionCode.DataInvalideException);
			link = link.replace("/tims/site/", "https://dsj.nrta.gov.cn/tims/site/");
			String linkPath = link + "&area=" + trObj.xpath("//td[1]/text()").toString();
			tvSet.add(linkPath);
		}
	}
}
			this.listProcess(page,tvSet,linkSet);
	
    } catch (IOException e) {
		e.printStackTrace();
	}
}

    private void listProcess(Page page,Set<String> linkSet,Set<String> urlKey){
    	try {
    		Set<TVPublicModel> tvSet = new HashSet<TVPublicModel>();
			for (String linkurl : linkSet) {
			Connection conn = HttpUtils.connent(linkurl);
			conn.get();
			Response rs = conn.response();
			String html = rs.body();
		if(StringUtil.isNotEmpty(html)){
			Document documentHtml = Jsoup.parse(html,"utf8");
			Html divHtml = new Html(documentHtml.toString());
	    	//报案机构
	    	 String produceCompany = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[2]/tbody/tr/td[1]/text()").toString();
	    	 if(StringUtils.isNotEmpty(produceCompany)){
	    		 produceCompany = HtmlUtil.delHTMLTag(produceCompany);
	    		 produceCompany = produceCompany.substring(5, produceCompany.length());
	    	 }
	    	 //公示日期
	    	 String pubMonth = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[2]/tbody/tr/td[2]/text()").toString();
	    	 if(StringUtils.isNotEmpty(pubMonth)){
	    		 pubMonth = pubMonth.substring(pubMonth.lastIndexOf("年") - 6, pubMonth.lastIndexOf("月"));
	    		 pubMonth = pubMonth.replace("年", "-").trim();
	    		 pubMonth = pubMonth.replace(" ", "");
				if (pubMonth.matches("^\\d{4}\\-\\d{1}$")) {
					pubMonth = pubMonth.replace("-", "-0");
				}
	    	 }
	    	 //许可证号
	    	 String licenseNum = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[2]/tbody/tr/td[3]/text()").toString();
	    	 if(StringUtils.isNotEmpty(licenseNum)){
	    		 licenseNum = HtmlUtil.delHTMLTag(licenseNum);
	    		 licenseNum = licenseNum.substring(6, licenseNum.length());
	    	 }
	    	 //剧名
	    	 String tvName = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[3]/tbody/tr[3]/td[1]/text()").toString();
	    	 if(StringUtils.isNotEmpty(tvName)){
	    		 tvName = HtmlUtil.delHTMLTag(tvName);
	    	 }
	    	 //题材
	    	 String subject = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[3]/tbody/tr[3]/td[2]/text()").toString();
	    	 if(StringUtils.isNotEmpty(subject)){
	    		 subject = HtmlUtil.delHTMLTag(subject);
	    	 }
	    	 //集数
	    	 String sets = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[3]/tbody/tr[3]/td[6]").toString();
	    	 if(StringUtils.isNotEmpty(sets)){
	    		 sets = HtmlUtil.delHTMLTag(sets);
	    	 }
	    	 //拍摄日期
	    	 String shoottime = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[3]/tbody/tr[3]/td[7]").toString();
	    	 if(StringUtils.isNotEmpty(shoottime)){
	    		 shoottime =HtmlUtil.delHTMLTag(shoottime);
	    		 shoottime = shoottime.replace(".", "-");
				if (shoottime.matches("^\\d{4}\\-\\d{1}$")) {
					shoottime = shoottime.replace("-", "-0");
				}
	    	 }
	    	 //制作周期
	    	 String makecycle = divHtml.xpath("//table[@cellpadding='0']/tbody/tr/td/table[3]/tbody/tr[3]/td[8]").toString();
	    	 if(StringUtils.isNotEmpty(makecycle)){
	    		 makecycle=makecycle.replace("个月", "");
	    		 makecycle = HtmlUtil.delHTMLTag(makecycle);
	    	 }
	    	 //剧情简介
	    	 String introduction = divHtml.xpath("//table[@cellpadding='0']/tbody/tr[@valign='top']/td").toString();
	    	 if(StringUtils.isNotEmpty(introduction)){
	    		 introduction = HtmlUtil.delHTMLTag(introduction);
	    	 }
	    	 //制作单位地区
	    	 String area = linkurl;
	    	 if(area.contains("area")){
	    		 area = area.substring(area.indexOf("area=") + 5);
	    	 }
	    	 tvSet.add(new TVPublicModel(Constants.TargetType.OTHER_TYPE_ID,pubMonth, tvName, area, produceCompany,
	    				subject, sets, shoottime, makecycle, introduction, licenseNum));
    	 
		}
	}
			if (!tvSet.isEmpty()){
				page.putField(TVPublicModel.class.getName(), tvSet);
				page.putField(URL_KEY,urlKey);
			}
		
	} catch (IOException e) {
		e.printStackTrace();
	}
    	 
}

	@Override
	protected String[] getStartRequestUrls() {
		//return new String[]{"https://dsj.nrta.gov.cn/tims/site/views/applications.shanty?appName=note"};
		return new String[]{start_url};
	}

	@Override
	protected String getSpiderType() {
		return Constants.SpiderTaskType.TV_PUBLIC;
	}

	@Override
	public String getPageDataType(Page page) {
		return Constants.PageDataType.SEARCH_LIST;
	}
}
